change retrieveExportWithContentIdentifier to take a list of ContentIdentifier

This partly fixes an issue where there are duplicate files in the
special remote, and the first file gets swapped with another duplicate,
or deleted. The swap case is fixed by this, the deleted case will need
other changes.

This makes retrieveExportWithContentIdentifier take a list of allowed
ContentIdentifier, same as storeExportWithContentIdentifier,
removeExportWithContentIdentifier, and
checkPresentExportWithContentIdentifier.

Of the special remotes that support importtree, borg is a special case
and does not use content identifiers, S3 I assume can't get mixed up
like this, directory certainly has the problem, and adb also appears to
have had the problem.

Sponsored-by: Graham Spencer on Patreon

diff --git a/Annex/Import.hs b/Annex/Import.hs
index d985e22cd5e7ea121ed79dda2a60ce6a6be744a0..1ab5c01abab4904fd59f39a22dfcf91a00a7ab36 100644 (file)
--- a/Annex/Import.hs
+++ b/Annex/Import.hs
@@ -600,7 +600,7 @@ importKeys remote importtreeconfig importcontent thirdpartypopulated importablec
                        let af = AssociatedFile (Just f)
                        let downloader p' tmpfile = do
                                _ <- Remote.retrieveExportWithContentIdentifier
-                                       ia loc cid (fromRawFilePath tmpfile)
+                                       ia loc [cid] (fromRawFilePath tmpfile)
                                        (Left k)
                                        (combineMeterUpdate p' p)
                                ok <- moveAnnex k af tmpfile
@@ -618,7 +618,7 @@ importKeys remote importtreeconfig importcontent thirdpartypopulated importablec
        doimportsmall cidmap db loc cid sz p = do
                let downloader tmpfile = do
                        (k, _) <- Remote.retrieveExportWithContentIdentifier
-                               ia loc cid (fromRawFilePath tmpfile)
+                               ia loc [cid] (fromRawFilePath tmpfile)
                                (Right (mkkey tmpfile))
                                p
                        case keyGitSha k of
@@ -641,7 +641,7 @@ importKeys remote importtreeconfig importcontent thirdpartypopulated importablec
                let af = AssociatedFile (Just f)
                let downloader tmpfile p = do
                        (k, _) <- Remote.retrieveExportWithContentIdentifier
-                               ia loc cid (fromRawFilePath tmpfile)
+                               ia loc [cid] (fromRawFilePath tmpfile)
                                (Right (mkkey tmpfile))
                                p
                        case keyGitSha k of

diff --git a/Remote/Adb.hs b/Remote/Adb.hs
index 715b8ab3a6ffe6d1996e05eabb13cd70712c9915..4f51887b4a9835cebe93df2aaf5a9eaf83919848 100644 (file)
--- a/Remote/Adb.hs
+++ b/Remote/Adb.hs
@@ -360,8 +360,8 @@ listImportableContentsM serial adir c = adbfind >>= \case
 -- connection is resonably fast, it's probably as good as
 -- git's handling of similar situations with files being modified while
 -- it's updating the working tree for a merge.
-retrieveExportWithContentIdentifierM :: AndroidSerial -> AndroidPath -> ExportLocation -> ContentIdentifier -> FilePath -> Either Key (Annex Key) -> MeterUpdate -> Annex (Key, Verification)
-retrieveExportWithContentIdentifierM serial adir loc cid dest gk _p = do
+retrieveExportWithContentIdentifierM :: AndroidSerial -> AndroidPath -> ExportLocation -> [ContentIdentifier] -> FilePath -> Either Key (Annex Key) -> MeterUpdate -> Annex (Key, Verification)
+retrieveExportWithContentIdentifierM serial adir loc cids dest gk _p = do
        case gk of
                Right mkkey -> do
                        go
@@ -374,9 +374,10 @@ retrieveExportWithContentIdentifierM serial adir loc cid dest gk _p = do
   where
        go = do
                retrieve' serial src dest
-               currcid <- getExportContentIdentifier serial adir loc
-               when (currcid /= Right (Just cid)) $
-                       giveup "the file on the android device has changed"
+               getExportContentIdentifier serial adir loc >>= \case
+                       Right (Just currcid)
+                               | any (currcid ==) cids -> return ()
+                       _ -> giveup "the file on the android device has changed"
        src = androidExportLocation adir loc
 
 storeExportWithContentIdentifierM :: AndroidSerial -> AndroidPath -> FilePath -> Key -> ExportLocation -> [ContentIdentifier] -> MeterUpdate -> Annex ContentIdentifier

diff --git a/Remote/Borg.hs b/Remote/Borg.hs
index bbfe166478259c25b8700bab64423656d0b2d128..2d347e35857317327633adec4b27e0c342acc0ae 100644 (file)
--- a/Remote/Borg.hs
+++ b/Remote/Borg.hs
@@ -371,7 +371,7 @@ checkPresentExportWithContentIdentifierM borgrepo _ loc _ = prompt $ liftIO $ do
                        , giveup $ "Unable to access borg repository " ++ locBorgRepo borgrepo
                        )
 
-retrieveExportWithContentIdentifierM :: BorgRepo -> ImportLocation -> ContentIdentifier -> FilePath -> Either Key (Annex Key) -> MeterUpdate -> Annex (Key, Verification)
+retrieveExportWithContentIdentifierM :: BorgRepo -> ImportLocation -> [ContentIdentifier] -> FilePath -> Either Key (Annex Key) -> MeterUpdate -> Annex (Key, Verification)
 retrieveExportWithContentIdentifierM borgrepo loc _ dest gk _ = do
        showOutput
        case gk of

diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index 5141e6a8f83696f7ffc50dd10efb1d8b1693dfd5..c0361b4d0f6bed345ca915ff0730ce73c4acd359 100644 (file)
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -394,12 +394,12 @@ mkContentIdentifier (IgnoreInodes ii) f st =
 -- versions of git-annex ignored inodes by default, treat two content
 -- idenfiers as the same if they differ only by one having the inode
 -- ignored.
-guardSameContentIdentifiers :: a -> ContentIdentifier -> Maybe ContentIdentifier -> a
+guardSameContentIdentifiers :: a -> [ContentIdentifier] -> Maybe ContentIdentifier -> a
 guardSameContentIdentifiers _ _ Nothing = giveup "file not found"
-guardSameContentIdentifiers cont old (Just new)
-       | new == old = cont
-       | ignoreinode new == old = cont
-       | new == ignoreinode old = cont
+guardSameContentIdentifiers cont olds (Just new)
+       | any (new ==) olds = cont
+       | any (ignoreinode new ==) olds = cont
+       | any (\old -> new == ignoreinode old) olds = cont
        | otherwise = giveup "file content has changed"
   where
        ignoreinode cid@(ContentIdentifier b) = 
@@ -417,7 +417,7 @@ importKeyM ii dir loc cid sz p = do
                { keySize = keySize kd <|> Just sz }
        currcid <- liftIO $ mkContentIdentifier ii absf
                =<< R.getSymbolicLinkStatus absf
-       guardSameContentIdentifiers (return (Just k)) cid currcid
+       guardSameContentIdentifiers (return (Just k)) [cid] currcid
   where
        f = fromExportLocation loc
        absf = dir P.</> f
@@ -427,8 +427,8 @@ importKeyM ii dir loc cid sz p = do
                , inodeCache = Nothing
                }
 
-retrieveExportWithContentIdentifierM :: IgnoreInodes -> RawFilePath -> CopyCoWTried -> ExportLocation -> ContentIdentifier -> FilePath -> Either Key (Annex Key) -> MeterUpdate -> Annex (Key, Verification)
-retrieveExportWithContentIdentifierM ii dir cow loc cid dest gk p =
+retrieveExportWithContentIdentifierM :: IgnoreInodes -> RawFilePath -> CopyCoWTried -> ExportLocation -> [ContentIdentifier] -> FilePath -> Either Key (Annex Key) -> MeterUpdate -> Annex (Key, Verification)
+retrieveExportWithContentIdentifierM ii dir cow loc cids dest gk p =
        case gk of
                Right mkkey -> do
                        go Nothing
@@ -474,7 +474,7 @@ retrieveExportWithContentIdentifierM ii dir cow loc cid dest gk p =
        
        -- Check before copy, to avoid expensive copy of wrong file
        -- content.
-       precheck cont = guardSameContentIdentifiers cont cid
+       precheck cont = guardSameContentIdentifiers cont cids
                =<< liftIO . mkContentIdentifier ii f
                =<< liftIO (R.getSymbolicLinkStatus f)
 
@@ -502,7 +502,7 @@ retrieveExportWithContentIdentifierM ii dir cow loc cid dest gk p =
 #else
                        =<< R.getSymbolicLinkStatus f
 #endif
-               guardSameContentIdentifiers cont cid currcid
+               guardSameContentIdentifiers cont cids currcid
 
        -- When copy-on-write was done, cannot check the handle that was
        -- copied from, but such a copy should run very fast, so
@@ -512,7 +512,7 @@ retrieveExportWithContentIdentifierM ii dir cow loc cid dest gk p =
        postcheckcow cont = do
                currcid <- liftIO $ mkContentIdentifier ii f
                        =<< R.getSymbolicLinkStatus f
-               guardSameContentIdentifiers cont cid currcid
+               guardSameContentIdentifiers cont cids currcid
 
 storeExportWithContentIdentifierM :: IgnoreInodes -> RawFilePath -> CopyCoWTried -> FilePath -> Key -> ExportLocation -> [ContentIdentifier] -> MeterUpdate -> Annex ContentIdentifier
 storeExportWithContentIdentifierM ii dir cow src _k loc overwritablecids p = do

diff --git a/Remote/Helper/ExportImport.hs b/Remote/Helper/ExportImport.hs
index 267c6d7ff6ae853e16191953e0d2fe3f2388510f..ddccda74dd8f3b2d78b3ace5e221af3814312ccc 100644 (file)
--- a/Remote/Helper/ExportImport.hs
+++ b/Remote/Helper/ExportImport.hs
@@ -359,14 +359,15 @@ adjustExportImport' isexport isimport r rs = do
                , giveup $ "exported content cannot be verified due to using the " ++ decodeBS (formatKeyVariety (fromKey keyVariety k)) ++ " backend"
                )
        
-       retrieveKeyFileFromImport dbv ciddbv k af dest p =
-               getkeycids ciddbv k >>= \case
-                       (cid:_) -> do
+       retrieveKeyFileFromImport dbv ciddbv k af dest p = do
+               cids <- getkeycids ciddbv k
+               if not (null cids)
+                       then do
                                l <- getfirstexportloc dbv k
-                               snd <$> retrieveExportWithContentIdentifier (importActions r) l cid dest (Left k) p
+                               snd <$> retrieveExportWithContentIdentifier (importActions r) l cids dest (Left k) p
                        -- In case a content identifier is somehow missing,
                        -- try this instead.
-                       [] -> if isexport
+                       else if isexport
                                then retrieveKeyFileFromExport dbv k af dest p
                                else giveup "no content identifier is recorded, unable to retrieve"
        

diff --git a/Remote/S3.hs b/Remote/S3.hs
index 749d7068023e2d33e61c43890d4cd39e8030fc2f..2b6580d69dfeb7a6370e1fcfb3d6d0c58d3c6cd4 100644 (file)
--- a/Remote/S3.hs
+++ b/Remote/S3.hs
@@ -649,8 +649,8 @@ mkImportableContentsVersioned info = build . groupfiles
                | otherwise =
                        i : removemostrecent mtime rest
 
-retrieveExportWithContentIdentifierS3 :: S3HandleVar -> Remote -> RemoteStateHandle -> S3Info -> ExportLocation -> ContentIdentifier -> FilePath -> Either Key (Annex Key) -> MeterUpdate -> Annex (Key, Verification)
-retrieveExportWithContentIdentifierS3 hv r rs info loc cid dest gk p =
+retrieveExportWithContentIdentifierS3 :: S3HandleVar -> Remote -> RemoteStateHandle -> S3Info -> ExportLocation -> [ContentIdentifier] -> FilePath -> Either Key (Annex Key) -> MeterUpdate -> Annex (Key, Verification)
+retrieveExportWithContentIdentifierS3 hv r rs info loc (cid:_) dest gk p =
        case gk of
                Right _mkkey -> do
                        k <- go Nothing
@@ -675,6 +675,7 @@ retrieveExportWithContentIdentifierS3 hv r rs info loc cid dest gk p =
                        return k
                Nothing -> giveup $ needS3Creds (uuid r)
        o = T.pack $ bucketExportLocation info loc
+retrieveExportWithContentIdentifierS3 _ _ _ _ _ [] _ _ _ = giveup "missing content identifier"
 
 {- Catch exception getObject returns when a precondition is not met,
  - and replace with a more understandable message for the user. -}

diff --git a/Types/Remote.hs b/Types/Remote.hs
index 97230f8090bd5db4769d4d67975e1c80ebeff9cd..e9e8a8c815f7dda44acc424d947df396f551010f 100644 (file)
--- a/Types/Remote.hs
+++ b/Types/Remote.hs
@@ -334,7 +334,7 @@ data ImportActions a = ImportActions
        -- Throws exception on failure to access the remote.
        , importKey :: Maybe (ImportLocation -> ContentIdentifier -> ByteSize -> MeterUpdate -> a (Maybe Key))
        -- Retrieves a file from the remote. Ensures that the file
-       -- it retrieves has the requested ContentIdentifier.
+       -- it retrieves has one of the requested ContentIdentifiers.
        --
        -- This has to be used rather than retrieveExport
        -- when a special remote supports imports, since files on such a
@@ -343,7 +343,7 @@ data ImportActions a = ImportActions
        -- Throws exception on failure.
        , retrieveExportWithContentIdentifier 
                :: ExportLocation
-               -> ContentIdentifier
+               -> [ContentIdentifier]
                -- file to write content to
                -> FilePath
                -- Either the key, or when it's not yet known, a callback